package com.suyunyou.spider.task;

import java.util.Date;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.suyunyou.manager.service.SpiderLinkService;
import com.suyunyou.spider.core.Counter;
import com.suyunyou.spider.data.FetcherLinkData;
import com.suyunyou.spider.data.LinkData;
import com.suyunyou.spider.model.Link;
import com.suyunyou.spider.utils.PageDownUtil;
import com.system.comm.utils.FrameSpringBeanUtil;
import com.system.comm.utils.FrameStringUtil;
import com.system.comm.utils.FrameTimeUtil;
import com.system.threadpool.FrameThreadAction;

/**
 * 爬取链接
 * @author yuejing
 * @date 2018年1月25日 下午1:28:06
 */
public class SpiderLinkTask extends FrameThreadAction {

	private static final long serialVersionUID = -8878461700070258924L;
	private static final Logger LOGGER = LoggerFactory.getLogger(SpiderLinkTask.class);
	private int downLinkErrorMaxNum;
	private int linkRepeatExtractHour;
	
	/**
	 * 构造函数
	 * @param downLinkErrorMaxNum		下载失败最大重试次数
	 * @param linkRepeatExtractHour		link重复提取间隔
	 */
	public SpiderLinkTask(int downLinkErrorMaxNum, int linkRepeatExtractHour) {
		this.downLinkErrorMaxNum = downLinkErrorMaxNum;
		this.linkRepeatExtractHour = linkRepeatExtractHour;
	}

	@Override
	protected void compute() {
		String link = FetcherLinkData.get(downLinkErrorMaxNum);
		try {
			if(FrameStringUtil.isEmpty(link)) {
				return;
			}
			
			Link l = LinkData.get(link);
			if(l != null && l.getFetcherContentTime() != null) {
				Date fcTime = FrameTimeUtil.addHours(l.getFetcherContentTime(), linkRepeatExtractHour);
				if(!link.equals(l.getSiteUrl()) && fcTime.compareTo(FrameTimeUtil.getTime()) > 0 && l.getDownErrorNum() == 0) {
					//在指定时间内，爬取过内容了则不重复爬取
					LinkData.addLinkFetcherContent(link, l.getContent(), false);
					Counter.plusPageDownSuccNum();
					LOGGER.error("在指定时间内，爬取过的内容[" + link + "]");
					return;
				} else if(l.getDownErrorNum().intValue() >= downLinkErrorMaxNum) {
					//失败次数达到最大值，则停止下载
					LOGGER.error("失败次数达到最大值，则停止下载[" + link + "]");
					return;
				}
			}
			String content = PageDownUtil.get(link);
			if(FrameStringUtil.isNotEmpty(content)) {
				LinkData.addLinkFetcherContent(link, content, true);
				Counter.plusPageDownSuccNum();
			} else {
				LOGGER.error("链接[" + link + "]提取内容为空: " + content);
			}
		} catch (Exception e) {
			LOGGER.error("链接[" + link + "]下载页面异常: " + e.getMessage());
			SpiderLinkService linkService = FrameSpringBeanUtil.getBean(SpiderLinkService.class);
			int downLinkErrorNum = linkService.updateDownErrorNum(link);
			if(downLinkErrorNum >= downLinkErrorMaxNum) {
				//移除下载
				//FetcherLinkData.remove(link);
				LOGGER.error("链接[" + link + "]下载页面异常达到最大重试次数，已移除下载");
			}
			Counter.plusPageDownErrorNum();
		}
	}
}